#!/usr/bin/env perl
###############################################################################
#
# Secuence fetcher script
# Search through the Nucleotide Genebank at NCBI for a particular secuence  									
# Developed by Marcos Oviedo [marcos.oviedo@fudepan.org.ar]
#
###############################################################################
#
# FuDePAN
# Copyright (C) Marcos Oviedo, 2008 - 2009, FuDePAN
#
#   sfetcher.pl is part of sequtils.
#
#   sequtils is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   sequtils is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with sequtils.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

use strict;
use warnings;
use BioUtils::SeqUtils;

require 5;

# Needed Modules
use Getopt::Long;

# unbuffer the outputs
select STDERR; $| = 1;
select STDOUT; $| = 1;

# Meaningful definitions
use constant SFETCHER_VERSION => "Fudepan sequence fetcher v1.1 (05/11/2009)";

my $eutilsURL = "http://www.ncbi.nlm.nih.gov/entrez/eutils";
my $esearchNucleotideURL = "$eutilsURL/esearch.fcgi?db=nucleotide&term=";
my $esummaryNucleotideURL = "$eutilsURL/esummary.fcgi?db=nucleotide&id=";
my $efetchNucleotideURL = "$eutilsURL/efetch.fcgi?db=nucleotide&retmode=xml&id=";
my $fudepanUserAgent = "FudepanUserAgent/1.0";
my $cacheDir = ".sfetcher_cache_dir";
my $cacheFile = "search_id_cache_file";


# working variables
my %options;
my $sequenceID;
my $searchID;
my $geneString;
my $downloadXML;
my $maxRetries;
my $briefMode;
my $cacheMode;
my $versionMode;
my $clearCache;
my $cacheSequenceIDFound;
my $cachePayloadFound;
my $translationMode;
my $helpMode;
my $callerScriptName;
my $sequenceOcurrences;
my $rawXMLContent;
my $sequence;
my $geneSequence;
my $matchedGene;
my $translationSequence;
my $translationCalculated;
my $lookedTranslationSequence;
my $httpTimeout;
my $cacheRoot;
my $workingCacheDir;
my $verbose;
my $tryIT;

#Grabbing out the caller script name
$callerScriptName = $0;

#Parsing out the provided arguments
#Check that enough arguments were provided
if ( @ARGV > 0 )
{
	#Display help if one of the needed arguments are missing
	GetOptions(
	    'sequence=s'    => \$sequenceID,
	    'feature:s'     => \$geneString,
		'translation'	=> \$translationMode,
	    'downloadXML' => \$downloadXML,
	    'maxretries:s' => \$maxRetries,
	    'briefmode' => \$briefMode,
	    'cachemode' => \$cacheMode,
	    'clearcache' => \$clearCache,
	    'query-timeout:s' => \$httpTimeout,
	    'version' => \$versionMode,
	    'verbose' => \$verbose,
	    'help|?' => \$helpMode,
	    ) or usage($callerScriptName, "Invalid Arguments were provided");
}
else
{
	usage($callerScriptName);
}

#Set functions verbosity if enabled
if( defined ($verbose) )
{
	set_verbosity(BioUtils::SeqUtils::ZTRUE);
}

#Version was requires called
if( defined ($versionMode) )
{
	print (SFETCHER_VERSION ."\n"); 
	exit(0);
}

#clearcache was requested
if ( defined ($clearCache) )
{
	fudepan_print("Clearing cache directory\n");

	#getting cache parent directory
	$cacheRoot = $ENV{HOME}; 	
	if ($cacheRoot eq "")
	{
		fudepan_print("HOME directory is not available, pointing cache dir to current directory\n");		
		$cacheRoot = get_working_dir();
	}

	$workingCacheDir = "$cacheRoot/$cacheDir";
	fudepan_print("Cache directory to delete is $workingCacheDir\n");

	#Checking first if cache directory exists
	if ( directory_exists($workingCacheDir) )
	{
		if ( delete_files($workingCacheDir) )
		{
			fudepan_print("Cache directory was sucessfully deleted\n");
		}
		else
		{
			fudepan_print("Cache directory couldn't be deleted\n");
		}
	}
	else
	{
		fudepan_print("Cache directory doesn't exists! (deleted already?) Exiting.\n");
	}

	exit(0)
}

#Help menu was called
if( defined ($helpMode) or not defined($sequenceID) )
{
	usage($callerScriptName); 
}

#BriefMode is not compatible with downloadXML and feature options
if ( defined($briefMode) and defined($geneString) )
{
	usage($callerScriptName, "\"briefmode\" cannot be used with \"feature\" option");
}

#check if maxretries contains a integer number
if ( defined($maxRetries) )
{
	if ( ($maxRetries !~ /^\d+$/) || ($maxRetries lt 0) )
	{ 
		usage($callerScriptName, "maxretries should contain a valid number and it should be greater than 0.");	
	}
	fudepan_print("Max retries set to $maxRetries\n");
}
else
{
	#if --maxretries was not provided, set it to 1
	$maxRetries = 1; 
}



##############################################################################
# Assigning default values and formating some of them
#

fudepan_print("Assigning default values\n");
$httpTimeout = 180 if not defined $httpTimeout;


##########################
# Real work starts here

#getting the ID to search for
if ( defined ($cacheMode) )
{
	fudepan_print("Cache mode enabled\n");
	
	#getting current directory
	$cacheRoot = $ENV{HOME}; 	
	if ($cacheRoot eq "")
	{
		fudepan_print("HOME directory is not available, pointing cache dir to current directory\n");		
		$cacheRoot = get_working_dir();
	}

	$workingCacheDir = "$cacheRoot/$cacheDir";
	fudepan_print("Cache directory set to $workingCacheDir\n");

	#Checking that the needed cache directory exists, and create it if this was was not the case
	if ( directory_exists($workingCacheDir) ne BioUtils::SeqUtils::ZTRUE )
	{
		fudepan_print("Cache directory does not exists, creating it\n");
        unless ( create_set_directory($workingCacheDir, 755) )
        {
            die "Seems to be a problem while creating/setting the following directory: $workingCacheDir\n".
                "Check if there are any special privileges applied to the directory\n";
        }	
        fudepan_print("Cache directory is empty, getting search ID from web\n");	
		$cacheSequenceIDFound = BioUtils::SeqUtils::ZFALSE;
	}
	else
	{
		#Checking sequence ID in cache
		fudepan_print("Getting from cache the ID to search for \n");
		if ( cache_get_search_id($workingCacheDir, $cacheFile, $sequenceID, \$searchID ) )
		{
			#Sequence found	on cache!
			fudepan_print("Search ID found on cache!\n");
			$cacheSequenceIDFound = BioUtils::SeqUtils::ZTRUE;

		}
		else
		{
			fudepan_print("Search ID cannot be found on cache, trying via web\n");
			$cacheSequenceIDFound = BioUtils::SeqUtils::ZFALSE;
		}

	}
}
else
{
	fudepan_print("Cache mode was not enabled\n");
}


#getting the ID to search for
if ( (not defined($cacheMode)) or ($cacheSequenceIDFound eq BioUtils::SeqUtils::ZFALSE) )
{
	for( $tryIT = 1; $tryIT <= $maxRetries; $tryIT++)
	{
		fudepan_print("Getting from web the ID to search for\n");
		unless( get_search_id($esearchNucleotideURL, $sequenceID, \$sequenceOcurrences, \$searchID, $httpTimeout ) )
		{
			die "Seems to be a problem while http-getting the SearchID of the sequence\n".
				"The number of ocurrences for the sequence $sequenceID was 0\n";
		}
		else
		{
			fudepan_print("ID found in try $tryIT\n");			
			last;
		}
	}
}

#updating search ID cache if necessary
if ( defined ($cacheMode) and ($cacheSequenceIDFound eq BioUtils::SeqUtils::ZFALSE) )
{
	fudepan_print("Updating cache with new Search ID\n");
    unless ( cache_update_search_id($workingCacheDir, $cacheFile, $sequenceID, $searchID) )
    {
        die "Seems to be a problem while updating cache search ID file ($cacheFile) at dir $workingCacheDir\n".
            "Check that you have enough privileges for the operation\n";
    }	
}

fudepan_print("ID found: $searchID\n");

#BriefMode (Summary mode) enabled
if ( defined ($briefMode) )
{
	#getting the ID to search for first from cache or web
	if ( defined ($cacheMode) )
	{
		fudepan_print("Getting summary of the sequence data from cache\n");
		if ( cache_get_eutil_xml_data($workingCacheDir, $esummaryNucleotideURL, $searchID, \$rawXMLContent ) )
		{
			fudepan_print("Summary data found on cache!\n");
			$cachePayloadFound = BioUtils::SeqUtils::ZTRUE;
		}
		else
		{
			fudepan_print("Summary data for Search ID $searchID cannot be found on cache, trying via web\n");
			$cachePayloadFound = BioUtils::SeqUtils::ZFALSE;
		}		
	}

	if ( (not defined ($cacheMode)) or ($cachePayloadFound eq BioUtils::SeqUtils::ZFALSE) )
	{
		for( $tryIT = 1; $tryIT <= $maxRetries; $tryIT++ )
		{
			#Getting summary of sequence data
			fudepan_print("Getting summary of the sequence data from web\n");
			unless( get_eutil_xml_data($esummaryNucleotideURL, $searchID, \$rawXMLContent ) )
			{
				die "Seems to be a problem while http-getting the summary of the data\n".
					"Check that the esummary CGI at NCBI is working properly\n";
			}
			else
			{
				fudepan_print("Summary of sequence found in try $tryIT\n");			
				last;
			}
		}
	}

	#updating search ID cache if necessary
	if ( defined ($cacheMode) and ($cachePayloadFound eq BioUtils::SeqUtils::ZFALSE) )
	{
		fudepan_print("Updating cache with $searchID summary file\n");
	    unless ( save_file($rawXMLContent, $workingCacheDir, $searchID . "_summary.xml") )
	    {
	        die "Seems to be a problem while saving content to cache for given file\n".
	            "Check that you have enough privileges for the operation\n";
	    }	
	}

	#Printing summary of the operation 
	fudepan_print("Printing summary details: \n");
	unless( print_esummary_xml_file($rawXMLContent) )
	{
		die "Seems to be a problem while parsing the summary details\n".
			"Check that the esummary CGI at NCBI returned a proper XML file\n";
	}
}
else
{
	#getting the ID to search for first from cache or web
	if ( defined ($cacheMode) )
	{
		fudepan_print("Getting sequence data content from cache\n");
		if ( cache_get_eutil_xml_data($workingCacheDir, $efetchNucleotideURL, $searchID, \$rawXMLContent ) )
		{
			fudepan_print("Sequence data found on cache!\n");
			$cachePayloadFound = BioUtils::SeqUtils::ZTRUE;
		}
		else
		{
			fudepan_print("Sequence data for Search ID $searchID cannot be found on cache, trying via web\n");
			$cachePayloadFound = BioUtils::SeqUtils::ZFALSE;
		}		
	}

	if ( (not defined ($cacheMode)) or ($cachePayloadFound eq BioUtils::SeqUtils::ZFALSE) )
	{
		#Getting content of sequence data from web
		fudepan_print("Getting sequence data from web\n");

		for( $tryIT = 1; $tryIT <= $maxRetries; $tryIT++ )
		{
			unless( get_eutil_xml_data($efetchNucleotideURL, $searchID, \$rawXMLContent ) )
			{
				die "Seems to be a problem while getting the content of the data\n".
					"Check that the efetch CGI at NCBI is working properly\n";
			}
			else
			{
				fudepan_print("Content of sequence found in try $tryIT\n");			
				last;
			}
		}
	}

	#updating search ID cache if necessary
	if ( defined ($cacheMode) and ($cachePayloadFound eq BioUtils::SeqUtils::ZFALSE) )
	{
		fudepan_print("Updating cache with $searchID contents\n");
	    unless ( save_file($rawXMLContent, $workingCacheDir, $searchID . "_content.xml") )
	    {
	        die "Seems to be a problem while saving content to cache for given file\n".
	            "Check that you have enough privileges for the operation\n";
	    }	
	}

	#Getting RAW sequence data
	fudepan_print("Getting secuence details\n");
	unless( get_efetch_sequence($rawXMLContent, \$sequence) )
	{
		die "Seems to be a problem while obtaining the sequence data\n".
			"Check that you provide an existing sequence data\n";
	}

	#Save XML mode
	if ( defined ($downloadXML) )
	{
		#Saving out the RAW XML content
		fudepan_print("Saving XML file\n");
		unless( save_file($rawXMLContent, get_current_dir(), "$searchID.xml") )
		{
			die "Seems to be a problem while saving the XML File\n".
				"Check that you have the rigth permissions for the operation\n";
		}
	}
	

	#Entering translation mode
	if ( defined($sequence )  and defined ($geneString) and defined($translationMode) )
	{
		#Calculating translation
		fudepan_print("Calculating Translation based on gene sequence \n");
		unless ( generate_translation_from_xml( $rawXMLContent, $sequence, $geneString, \$translationCalculated ) )
		{				
			die "Seems to be a problem while calculating the translation sequence data\n".
				"Check that you provide an existing gene and nucleotide\n";
		}
		
		fudepan_print("Translation generated based on sequence of $geneString gene\n");  
		
		
		#Looking for the translation translation gene sequence data 
		if ( defined( $translationCalculated ) and defined ( $geneString ) )
		{
			fudepan_print("Trying to found the translation sequence using absolute matching\n");
			#At first look for the translation using absolute match mode
			unless( get_efetch_translation( $rawXMLContent, $sequence, $geneString, \$lookedTranslationSequence, BioUtils::SeqUtils::ZFALSE, \$matchedGene ) )
			{				
				fudepan_print("Translation sequence not found using absolute matching, now trying using fuzzy matching\n");
				#If gene was not found, look for it using fuzzy matching 
				if( get_efetch_translation( $rawXMLContent, $sequence, $geneString, \$lookedTranslationSequence, BioUtils::SeqUtils::ZTRUE, \$matchedGene ) )
				{			
			  		fudepan_print("Translation found using fuzzy matching on gene: \"$matchedGene\"\n");  	
				}				
			}				
		}	


		#Now is the NBCI over translation validation logic
		#Asummint that $translationCalculated will be there and it will contain proper values

		#First Case: Translation at NBCI was not found, using the calculated translation
		if ( not defined ($lookedTranslationSequence) ) 
		{
			$translationSequence = $translationCalculated;
		}
		else
		{
			#Boths translation are equal? using the calculated one
			if ( $translationCalculated eq $lookedTranslationSequence )
			{
				$translationSequence = $translationCalculated;
			}
			#Check if the translations are compatible
			elsif ( are_translations_compatible( $translationCalculated, $lookedTranslationSequence)  )
			{				
				fudepan_print("Using Translation calculated. The one obtained at NCBI is compatible with one calculated\n");  	
				$translationSequence = $translationCalculated;
			}
			#Something horrible happens here
			else
			{	
				die "Seems to be a problem while calculating the translation sequence data\n".
					"Check that you provide an existing gene and nucleotide\n";
			}			
		}
		
		
		#Now finally printing the translation
		print "$translationSequence\n";
			}

	#Entering secuence only mode
	elsif ( defined( $geneString ) )
	{
		fudepan_print("Printing gene secuence details\n");
		unless( get_efetch_sequence_with_gene( $rawXMLContent, $sequence, $geneString, \$geneSequence ) )
		{
			fudepan_print("Gene sequence not found using absolute matching, now trying using fuzzy matching\n");
			#If gene was not found, look for it using fuzzy matching 
			unless( fuzzy_get_efetch_sequence_with_gene( $rawXMLContent, $sequence, $geneString, \$geneSequence, \$matchedGene ) )
			{
				die "Seems to be a problem while parsing the sequence data\n".
					"Check that you provide an existing sequence data\n";
			}
	  		fudepan_print("Gene sequence found using fuzzy matching on gene: \"$matchedGene\"\n");
		}
		print "$geneSequence\n";
	}
	#Printing sequence data 
	else
	{
		fudepan_print("Printing secuence details\n");
		print "$sequence\n";
	}
}


fudepan_print("Success! data was obtained successfully\n");



#####################################################################
# Helper Functions
#####################################################################


#######
# Print usage information
# Parameter: script name
# Returns: nothing
#
sub usage 
{
    my ($caller, $stringToDisplay) = @_;

	if ( defined ($stringToDisplay) )
	{
		print STDERR ("Problem detected: $stringToDisplay\n");	
	} 

	#printing help
	die << "EOF";
Secuence fetcher script
Developed by Marcos Oviedo [marcos.oviedo at fudepan.org.ar]
Usage: $caller [options] (please use quotes for "long inputs")
Required options: --sequence		Sequence to look for (e.g AY207443)
	                                                  
Optional options: --feature		Gene to look for inside the sequence (e.g aagg)
       	          --brief		Brief Mode	
       	          --cachemode		Cache Mode
       	          --clearcache		Clear cache directory if present				  
       	          --query-timeout	Timeout for HTTP GET queries (default to 180 sec)
       	          --maxretries	Maximum number of http retries
       	          --download		Download XML file resulted from operations
       	          --version		Print version
       	          --verbose		Verbose output
       	          --help		Help Menu
EOF
}

